from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk

#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_1%", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_2%", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_3%", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_4%", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_5%", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_6%", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_7%", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_8%", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_9%", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_10%", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_20%", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_30%", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_40%", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_50%", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_60%", "tonsOfIndexesMaking_20140920/60PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/60PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot9_70%", "tonsOfIndexesMaking_20140920/70PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/70PercentageIndex_UPP-5_ptPowToDot9_20140920/index.lex.0.0") )

file1 = sys.argv[1]
file2 = sys.argv[2]
file3 = sys.argv[3]
tuple = (file1,file2,file3)

needToDealWithTripleList = []
needToDealWithTripleList.append(tuple)


#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_20%", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_30%", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_40%", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_50%", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_60%", "tonsOfIndexesMaking_20140920/60PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/60PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_70%", "tonsOfIndexesMaking_20140920/70PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/70PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )


#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_1%", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_2%", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_3%", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_4%", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_5%", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_6%", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_7%", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_8%", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_9%", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_10%", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_20%", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_30%", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_40%", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowToDot5_50%", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowToDot5_20140920/index.lex.0.0") )

#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_1%", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/1PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_2%", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/2PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_3%", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/3PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_4%", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/4PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_5%", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/5PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_6%", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/6PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_7%", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/7PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_8%", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/8PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_9%", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/9PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
#needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_10%", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/10PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
##needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_20%", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/20PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
##needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_30%", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/30PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
##needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_40%", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/40PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )
##needToDealWithTripleList.append( ("termUpperBoundsOfListLength_20141001_ptPowTo1_50%", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0_Original", "tonsOfIndexesMaking_20140920/50PercentageIndex_UPP-5_ptPowTo1_20140919/index.lex.0.0") )

basePath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/"

for tuple in needToDealWithTripleList:
    (ifn0,ifn,ofn) = tuple
    termUpperBoundDict = {}
    ifn0 = basePath + ifn0
    ifh0 = open(ifn0,"r")
        
    ifn = basePath + ifn
    ifh = open(ifn,"r")
    
    ofn = basePath + ofn
    ofh = open(ofn,"wb")
    
    print "Reading",ifn0
    print "Reading",ifn
    print "Writing",ofn
    
    #step1:
    #input format
    #0 0 128 1.42934
    #1 00 105 2.65071
    for l in ifh0.readlines():
        le = l.strip().split(" ")
        term = le[1]
        termUpperBound = float(le[3])
        termUpperBoundDict[term] = termUpperBound
    print "len(termUpperBoundDict):",len(termUpperBoundDict)
    
    #step2:
    statinfo = os.stat(ifn)
    fileSize = statinfo.st_size
    print "currentInputFileName: ",ifn
    print "file size:",fileSize
    numOfBytesRead = 0
    termCounter = 0
    while numOfBytesRead < fileSize:
        
        byteString_numOfLayers = ifh.read(4)
        byteString_termLen = ifh.read(4)
        
        (numOfLayers,) = unpack( "i", byteString_numOfLayers)
        (termLen,) = unpack( "i", byteString_termLen)
        
        byteString_term = ifh.read(termLen)
        
        byteString_numOfDocs = ifh.read(4)
        (numOfDocs,) = unpack( "i", byteString_numOfDocs)
        
        byteString_numOfChunks = ifh.read(4)
        (numOfChunks,) = unpack( "i", byteString_numOfChunks)
        
        byteString_numOfChunksLastBlock = ifh.read(4)
        (numOfChunksLastBlock,) = unpack( "i", byteString_numOfChunksLastBlock)
        
        byteString_numOfBlocks = ifh.read(4)
        (numOfBlocks,) = unpack( "i", byteString_numOfBlocks)
        
        byteString_blockNum = ifh.read(4)
        (blockNum,) = unpack( "i", byteString_blockNum)
        
        byteString_chunkNum = ifh.read(4)
        (chunkNum,) = unpack( "i", byteString_chunkNum)
        
        byteString_scoreThreshold = ifh.read(4)
        (scoreThreshold,) = unpack( "f", byteString_scoreThreshold)
        
        byteString_externalIndexOffset = ifh.read(8) # for the externalIndexOffset 
        
        numOfBytesRead += 4*2 + termLen + 4*9
        # print termCounter,byteString_term,numOfDocs,scoreThreshold,termUpperBoundDict[byteString_term],numOfBytesRead
        termCounter += 1
        
        ofh.write(byteString_numOfLayers)
        ofh.write(byteString_termLen)
        ofh.write(byteString_term)
        ofh.write(byteString_numOfDocs)
        ofh.write(byteString_numOfChunks)
        ofh.write(byteString_numOfChunksLastBlock)
        ofh.write(byteString_numOfBlocks)
        ofh.write(byteString_blockNum)
        ofh.write(byteString_chunkNum)
        ofh.write(pack("1f", termUpperBoundDict[byteString_term]))
        ofh.write(byteString_externalIndexOffset)
    
    print "Overall:"
    print "ifn0:",ifn0
    print "ifn:",ifn
    print "ofn:",ofn
    print "# of terms in the lexicon:",termCounter
    print "DONE."
    print
    ifh0.close()
    ifh.close()
    ofh.close()
